


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  <meta name="generator" content="Docutils 0.17.1: http://docutils.sourceforge.net/" />

  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>Installation &mdash; Torch-TensorRT v2.10.0.dev0+2e6843e documentation</title>
  

  
  
  
  

  

  
  
    

  

  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-binder.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-dataframe.css" type="text/css" />
  <link rel="stylesheet" href="../_static/sg_gallery-rendered-html.css" type="text/css" />
  <link rel="stylesheet" href="../_static/collapsible-lists/css/tree_view.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../_static/css/custom.css" type="text/css" />
    <link rel="index" title="Index" href="../genindex.html" />
    <link rel="search" title="Search" href="../search.html" />
    <link rel="next" title="Torch-TensorRT in JetPack" href="jetpack.html" />
    <link rel="prev" title="Torch-TensorRT" href="../index.html" />
  <!-- Google Tag Manager -->
    <script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':
    new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0],
    j=d.createElement(s),dl=l!='dataLayer'?'&l='+l:'';j.async=true;j.src=
    'https://www.googletagmanager.com/gtm.js?id='+i+dl;f.parentNode.insertBefore(j,f);
    })(window,document,'script','dataLayer','');</script>
    <!-- End Google Tag Manager -->
  

  
  <script src="../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
  <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>

          <li class="main-menu-item">
          <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Learn
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/get-started">
                  <span class=dropdown-title>Get Started</span>
                  <p>Run PyTorch locally or get started quickly with one of the supported cloud platforms</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials">
                  <span class="dropdown-title">Tutorials</span>
                  <p>Whats new in PyTorch tutorials</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials/beginner/basics/intro.html">
                  <span class="dropdown-title">Learn the Basics</span>
                  <p>Familiarize yourself with PyTorch concepts and modules</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials/recipes/recipes_index.html">
                  <span class="dropdown-title">PyTorch Recipes</span>
                  <p>Bite-size, ready-to-deploy PyTorch code examples</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tutorials/beginner/introyt.html">
                  <span class="dropdown-title">Intro to PyTorch - YouTube Series</span>
                  <p>Master PyTorch basics with our engaging YouTube tutorial series</p>
                </a>
              </div>
            </div>
          </li>

          <li>
          <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Ecosystem
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class="dropdown-title">Tools</span>
                  <p>Learn about the tools and frameworks in the PyTorch Ecosystem</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/#community-module">
                  <span class=dropdown-title>Community</span>
                  <p>Join the PyTorch developer community to contribute, learn, and get your questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://discuss.pytorch.org/" target="_blank">
                  <span class=dropdown-title>Forums</span>
                  <p>A place to discuss PyTorch code, issues, install, research</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/resources">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem/contributor-awards-2024">
                  <span class="dropdown-title">Contributor Awards - 2024</span>
                  <p>Award winners announced at this year's PyTorch Conference</p>
                </a>
              </div>
            </div>
          </li>

          <li>
          <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Edge
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/edge">
                  <span class="dropdown-title">About PyTorch Edge</span>
                  <p>Build innovative and privacy-aware AI experiences for edge devices</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/executorch-overview">
                  <span class="dropdown-title">ExecuTorch</span>
                  <p>End-to-end solution for enabling on-device inference capabilities across mobile and edge devices</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/executorch/stable/index.html">
                  <span class="dropdown-title">ExecuTorch Docs</span>
                </a>
              </div>
            </div>  
          </li>

          <li class="main-menu-item">
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Docs
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/docs/stable/index.html">
                  <span class="dropdown-title">PyTorch</span>
                  <p>Explore the documentation for comprehensive guidance on how to use PyTorch</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/pytorch-domains">
                  <span class="dropdown-title">PyTorch Domains</span>
                  <p>Read the PyTorch Domains documentation to learn more about domain-specific libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                Blogs & News 
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/blog/">
                  <span class="dropdown-title">PyTorch Blog</span>
                  <p>Catch up on the latest technical news and happenings</p>
                </a>
                 <a class="nav-dropdown-item" href="https://pytorch.org/community-blog">
                  <span class="dropdown-title">Community Blog</span>
                  <p>Stories from the PyTorch ecosystem</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/videos">
                  <span class="dropdown-title">Videos</span>
                  <p>Learn about the latest PyTorch tutorials, new, and more </p>
                <a class="nav-dropdown-item" href="https://pytorch.org/community-stories">
                  <span class="dropdown-title">Community Stories</span>
                  <p>Learn how our community solves real, everyday machine learning problems with PyTorch</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/events">
                  <span class="dropdown-title">Events</span>
                  <p>Find events, webinars, and podcasts</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/newsletter">
                  <span class="dropdown-title">Newsletter</span>
                  <p>Stay up-to-date with the latest updates</p>
                </a>
            </div>
          </li>

          <li>
            <div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
              <a class="with-down-arrow">
                About
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/foundation">
                  <span class="dropdown-title">PyTorch Foundation</span>
                  <p>Learn more about the PyTorch Foundation</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/governing-board">
                  <span class="dropdown-title">Governing Board</span>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/credits">
                  <span class="dropdown-title">Cloud Credit Program</span>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/tac">
                  <span class="dropdown-title">Technical Advisory Council</span>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/staff">
                  <span class="dropdown-title">Staff</span>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/contact-us">
                  <span class="dropdown-title">Contact Us</span>
                </a>
              </div>
            </div>
          </li>

          <li class="main-menu-item">
            <div class="no-dropdown">
              <a href="https://pytorch.org/join" data-cta="join">
                Become a Member
              </a>
            </div>
          </li>
          <li>
           <div class="main-menu-item">
             <a href="https://github.com/pytorch/pytorch" class="github-icon">
             </a>
           </div>
          </li>
          <!--- TODO: This block adds the search icon to the nav bar. We will enable it later. 
          <li>
            <div class="main-menu-item">
             <a href="https://github.com/pytorch/pytorch" class="search-icon">
             </a>
            </div>
          </li>
          --->
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>
  </div>
</div>

<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  v2.10.0.dev0+2e6843e
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
            
            
              
            
            
              <p class="caption" role="heading"><span class="caption-text">Getting Started</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="current reference internal" href="#">Installation</a></li>
<li class="toctree-l1"><a class="reference internal" href="jetpack.html">Torch-TensorRT in JetPack</a></li>
<li class="toctree-l1"><a class="reference internal" href="quick_start.html">Quick Start</a></li>
<li class="toctree-l1"><a class="reference internal" href="capture_and_replay.html">Introduction</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">User Guide</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../user_guide/torch_tensorrt_explained.html">Torch-TensorRT Explained</a></li>
<li class="toctree-l1"><a class="reference internal" href="../user_guide/dynamic_shapes.html">Dynamic shapes with Torch-TensorRT</a></li>
<li class="toctree-l1"><a class="reference internal" href="../user_guide/saving_models.html">Saving models compiled with Torch-TensorRT</a></li>
<li class="toctree-l1"><a class="reference internal" href="../user_guide/runtime.html">Deploying Torch-TensorRT Programs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../user_guide/using_dla.html">DLA</a></li>
<li class="toctree-l1"><a class="reference internal" href="../user_guide/mixed_precision.html">Compile Mixed Precision models with Torch-TensorRT</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Tutorials</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_compile_advanced_usage.html">Torch Compile Advanced Usage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/vgg16_ptq.html">Deploy Quantized Models using Torch-TensorRT</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/engine_caching_example.html">Engine Caching</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/engine_caching_bert_example.html">Engine Caching (BERT)</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/refit_engine_example.html">Refitting Torch-TensorRT Programs with New Weights</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/serving_torch_tensorrt_with_triton.html">Serving a Torch-TensorRT model with Triton</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_export_cudagraphs.html">Torch Export with Cudagraphs</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/converter_overloading.html">Overloading Torch-TensorRT Converters with Custom Converters</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/custom_kernel_plugins.html">Using Custom Kernels within TensorRT Engines with Torch-TensorRT</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/auto_generate_converters.html">Automatically Generate a Converter for a Custom Kernel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/auto_generate_plugins.html">Automatically Generate a Plugin for a Custom Kernel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/mutable_torchtrt_module_example.html">Mutable Torch TensorRT Module</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/weight_streaming_example.html">Weight Streaming</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/pre_allocated_output_example.html">Pre-allocated output buffer</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Dynamo Frontend</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../dynamo/torch_compile.html">TensorRT Backend for <code class="docutils literal notranslate"><span class="pre">torch.compile</span></code></a></li>
<li class="toctree-l1"><a class="reference internal" href="../dynamo/dynamo_export.html">Compiling Exported Programs with Torch-TensorRT</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">TorchScript Frontend</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../ts/creating_torchscript_module_in_python.html">Creating a TorchScript Module</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ts/creating_torchscript_module_in_python.html#working-with-torchscript-in-python">Working with TorchScript in Python</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ts/creating_torchscript_module_in_python.html#saving-torchscript-module-to-disk">Saving TorchScript Module to Disk</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ts/getting_started_with_python_api.html">Using Torch-TensorRT in Python</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ts/getting_started_with_cpp_api.html">Using Torch-TensorRT in  C++</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ts/ptq.html">Post Training Quantization (PTQ)</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">FX Frontend</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../fx/getting_started_with_fx_path.html">Torch-TensorRT (FX Frontend) User Guide</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Model Zoo</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_compile_resnet_example.html">Compiling ResNet with dynamic shapes using the <cite>torch.compile</cite> backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_compile_transformers_example.html">Compiling BERT using the <cite>torch.compile</cite> backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_compile_stable_diffusion.html">Compiling Stable Diffusion model using the <cite>torch.compile</cite> backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/compile_hf_models.html">Compiling LLM models from Huggingface</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_compile_gpt2.html">Compiling GPT2 using the Torch-TensorRT <code class="docutils literal notranslate"><span class="pre">torch.compile</span></code> frontend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_export_sam2.html">Compiling SAM2 using the dynamo backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/_rendered_examples/dynamo/torch_export_flux_dev.html">Compiling FLUX.1-dev model using the Torch-TensorRT dynamo backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tutorials/notebooks.html">Legacy notebooks</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Python API Documentation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../py_api/torch_tensorrt.html">torch_tensorrt</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/dynamo.html">torch_tensorrt.dynamo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/logging.html">torch_tensorrt.logging</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/fx.html">torch_tensorrt.fx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/ts.html">torch_tensorrt.ts</a></li>
<li class="toctree-l1"><a class="reference internal" href="../py_api/ptq.html">torch_tensorrt.ts.ptq</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">C++ API Documentation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/torch_tensort_cpp.html">Torch-TensorRT C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt.html">Namespace torch_tensorrt</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt__logging.html">Namespace torch_tensorrt::logging</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_cpp_api/namespace_torch_tensorrt__torchscript.html">Namespace torch_tensorrt::torchscript</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">CLI Documentation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../cli/torchtrtc.html">torchtrtc</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Contributor Documentation</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../contributors/system_overview.html">System Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/dynamo_converters.html">Writing Dynamo Converters</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/writing_dynamo_aten_lowering_passes.html">Writing Dynamo ATen Lowering Passes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/ts_converters.html">Writing TorchScript Converters</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/useful_links.html">Useful Links for Torch-TensorRT Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="../contributors/resource_management.html">Resource Management</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Indices</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../indices/supported_ops.html">Operators Supported</a></li>
</ul>

            
          
        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
      <li>Installation</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
            
            <a href="../_sources/getting_started/installation.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>
          
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          <!-- Google Tag Manager (noscript) -->
          <noscript><iframe src="https://www.googletagmanager.com/ns.html?id="
          height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
          <!-- End Google Tag Manager (noscript) -->
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <section id="installation">
<span id="id1"></span><h1>Installation<a class="headerlink" href="#installation" title="Permalink to this heading">¶</a></h1>
<section id="precompiled-binaries">
<h2>Precompiled Binaries<a class="headerlink" href="#precompiled-binaries" title="Permalink to this heading">¶</a></h2>
<p>Torch-TensorRT 2.x is centered primarily around Python. As such, precompiled releases can be found on <a class="reference external" href="https://pypi.org/project/torch-tensorrt/">pypi.org</a></p>
<section id="dependencies">
<h3>Dependencies<a class="headerlink" href="#dependencies" title="Permalink to this heading">¶</a></h3>
<p>You need to have CUDA, PyTorch, and TensorRT (python package is sufficient) installed to use Torch-TensorRT</p>
<blockquote>
<div><ul class="simple">
<li><p><a class="reference external" href="https://developer.nvidia.com/cuda">https://developer.nvidia.com/cuda</a></p></li>
<li><p><a class="reference external" href="https://pytorch.org">https://pytorch.org</a></p></li>
</ul>
</div></blockquote>
</section>
<section id="installing-torch-tensorrt">
<h3>Installing Torch-TensorRT<a class="headerlink" href="#installing-torch-tensorrt" title="Permalink to this heading">¶</a></h3>
<p>You can install the python package using</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>torch<span class="w"> </span>torch-tensorrt<span class="w"> </span>tensorrt
</pre></div>
</div>
<p>Packages are uploaded for Linux on x86 and Windows</p>
</section>
<section id="installing-torch-tensorrt-for-a-specific-cuda-version">
<h3>Installing Torch-TensorRT for a specific CUDA version<a class="headerlink" href="#installing-torch-tensorrt-for-a-specific-cuda-version" title="Permalink to this heading">¶</a></h3>
<p>Similar to PyTorch, Torch-TensorRT has builds compiled for different versions of CUDA. These are distributed on PyTorch’s package index</p>
<p>For example CUDA 11.8</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>torch<span class="w"> </span>torch-tensorrt<span class="w"> </span>tensorrt<span class="w"> </span>--extra-index-url<span class="w"> </span>https://download.pytorch.org/whl/cu118
</pre></div>
</div>
</section>
<section id="installing-nightly-builds">
<h3>Installing Nightly Builds<a class="headerlink" href="#installing-nightly-builds" title="Permalink to this heading">¶</a></h3>
<p>Torch-TensorRT distributed nightlies targeting the PyTorch nightly. These can be installed from the PyTorch nightly package index (separated by CUDA version)</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>--pre<span class="w"> </span>torch<span class="w"> </span>torch-tensorrt<span class="w"> </span>tensorrt<span class="w"> </span>--extra-index-url<span class="w"> </span>https://download.pytorch.org/whl/nightly/cu130
</pre></div>
</div>
</section>
<section id="c-precompiled-binaries-torchscript-only">
<span id="bin-dist"></span><h3>C++ Precompiled Binaries (TorchScript Only)<a class="headerlink" href="#c-precompiled-binaries-torchscript-only" title="Permalink to this heading">¶</a></h3>
<p>Precompiled tarballs for releases are provided here: <a class="reference external" href="https://github.com/pytorch/TensorRT/releases">https://github.com/pytorch/TensorRT/releases</a></p>
</section>
</section>
<section id="compiling-from-source">
<span id="compile-from-source"></span><h2>Compiling From Source<a class="headerlink" href="#compiling-from-source" title="Permalink to this heading">¶</a></h2>
<section id="building-on-linux">
<h3>Building on Linux<a class="headerlink" href="#building-on-linux" title="Permalink to this heading">¶</a></h3>
<section id="installing-deps">
<span id="id2"></span><h4>Dependencies<a class="headerlink" href="#installing-deps" title="Permalink to this heading">¶</a></h4>
<ul>
<li><p>Torch-TensorRT is built with <strong>Bazel</strong>, so begin by installing it.</p>
<blockquote>
<div><ul class="simple">
<li><p>The easiest way is to install bazelisk using the method of your choosing <a class="reference external" href="https://github.com/bazelbuild/bazelisk">https://github.com/bazelbuild/bazelisk</a></p></li>
<li><p>Otherwise you can use the following instructions to install binaries <a class="reference external" href="https://docs.bazel.build/versions/master/install.html">https://docs.bazel.build/versions/master/install.html</a></p></li>
<li><p>Finally if you need to compile from source (e.g. aarch64 until bazel distributes binaries for the architecture) you can use these instructions</p></li>
</ul>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span><span class="nb">export</span><span class="w"> </span><span class="nv">BAZEL_VERSION</span><span class="o">=</span><span class="k">$(</span>cat<span class="w"> </span>&lt;PATH_TO_TORCHTRT_ROOT&gt;/.bazelversion<span class="k">)</span>
mkdir<span class="w"> </span>bazel
<span class="nb">cd</span><span class="w"> </span>bazel
curl<span class="w"> </span>-fSsL<span class="w"> </span>-O<span class="w"> </span>https://github.com/bazelbuild/bazel/releases/download/<span class="nv">$BAZEL_VERSION</span>/bazel-<span class="nv">$BAZEL_VERSION</span>-dist.zip
unzip<span class="w"> </span>bazel-<span class="nv">$BAZEL_VERSION</span>-dist.zip
bash<span class="w"> </span>./compile.sh
cp<span class="w"> </span>output/bazel<span class="w"> </span>/usr/local/bin/
</pre></div>
</div>
</div></blockquote>
</li>
<li><p>You will also need to have <strong>CUDA</strong> installed on the system (or if running in a container, the system must have the CUDA driver installed and the container must have CUDA)</p>
<blockquote>
<div><ul class="simple">
<li><p>Specify your CUDA version here if not the version used in the branch being built: <a class="reference external" href="https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L46">https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L46</a></p></li>
</ul>
</div></blockquote>
</li>
<li><p>The correct <strong>LibTorch</strong> and <strong>TensorRT</strong> versions will be pulled down for you by bazel.</p>
<blockquote>
<div><p>NOTE: By default bazel will pull the latest nightly from pytorch.org. For building main, this is usually sufficient however if there is a specific PyTorch you are targeting,
edit these locations with updated URLs/paths:</p>
<ul class="simple">
<li><p><a class="reference external" href="https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L53C1-L53C1">https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L53C1-L53C1</a></p></li>
</ul>
</div></blockquote>
</li>
<li><p><strong>TensorRT</strong> is not required to be installed on the system to build Torch-TensorRT, in fact this is preferable to ensure reproducible builds. If versions other than the default are needed
point the WORKSPACE file to the URL of the tarball or download the tarball for TensorRT from <a class="reference external" href="https://developer.nvidia.com">https://developer.nvidia.com</a> and update the paths in the WORKSPACE file here <a class="reference external" href="https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L71">https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L71</a></p>
<blockquote>
<div><p>For example:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">http_archive</span><span class="p">(</span>
    <span class="n">name</span> <span class="o">=</span> <span class="s2">&quot;tensorrt&quot;</span><span class="p">,</span>
    <span class="n">build_file</span> <span class="o">=</span> <span class="s2">&quot;@//third_party/tensorrt/archive:BUILD&quot;</span><span class="p">,</span>
    <span class="n">sha256</span> <span class="o">=</span> <span class="s2">&quot;&lt;TENSORRT SHA256&gt;&quot;</span><span class="p">,</span> <span class="c1"># Optional but recommended</span>
    <span class="n">strip_prefix</span> <span class="o">=</span> <span class="s2">&quot;TensorRT-&lt;TENSORRT VERSION&gt;&quot;</span><span class="p">,</span>
    <span class="n">urls</span> <span class="o">=</span> <span class="p">[</span>
        <span class="s2">&quot;https://developer.nvidia.com/downloads/compute/machine-learning/tensorrt/&lt;TENSORRT DOWNLOAD PATH&gt;&quot;</span><span class="p">,</span>
        <span class="c1"># OR</span>
        <span class="s2">&quot;file:///&lt;ABSOLUTE PATH TO FILE&gt;/TensorRT-&lt;TENSORRT VERSION&gt;.Linux.x86_64-gnu.cuda-&lt;CUDA VERSION&gt;.tar.gz&quot;</span>
    <span class="p">],</span>
<span class="p">)</span>
</pre></div>
</div>
<p>Remember at runtime, these libraries must be added to your <code class="docutils literal notranslate"><span class="pre">LD_LIBRARY_PATH</span></code> explicitly</p>
</div></blockquote>
</li>
</ul>
<p>If you have a local version of TensorRT installed, this can be used as well by commenting out the above lines and uncommenting the following lines <a class="reference external" href="https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L114C1-L124C3">https://github.com/pytorch/TensorRT/blob/4e5b0f6e860910eb510fa70a76ee3eb9825e7a4d/WORKSPACE#L114C1-L124C3</a></p>
</section>
<section id="building-the-package">
<h4>Building the Package<a class="headerlink" href="#building-the-package" title="Permalink to this heading">¶</a></h4>
<p>Once the WORKSPACE has been configured properly, all that is required to build torch-tensorrt is the following command</p>
<blockquote>
<div><div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>--pre<span class="w"> </span>.<span class="w"> </span>--extra-index-url<span class="w"> </span>https://download.pytorch.org/whl/nightly/cu130
</pre></div>
</div>
</div></blockquote>
<p>If you use the <code class="docutils literal notranslate"><span class="pre">uv</span></code> (<a class="reference external" href="https://docs.astral.sh/uv/">https://docs.astral.sh/uv/</a>) tool to manage python and your projects, the command is slightly simpler</p>
<blockquote>
<div><div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>uv<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>-e<span class="w"> </span>.
</pre></div>
</div>
</div></blockquote>
<p>To build the wheel file</p>
<blockquote>
<div><div class="highlight-sh notranslate"><div class="highlight"><pre><span></span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>wheel<span class="w"> </span>--no-deps<span class="w"> </span>--pre<span class="w"> </span>.<span class="w"> </span>--extra-index-url<span class="w"> </span>https://download.pytorch.org/whl/nightly/cu130<span class="w"> </span>-w<span class="w"> </span>dist
</pre></div>
</div>
</div></blockquote>
</section>
<section id="additional-build-options">
<h4>Additional Build Options<a class="headerlink" href="#additional-build-options" title="Permalink to this heading">¶</a></h4>
<p>Some features in the library are optional and allow builds to be lighter or more portable.</p>
<section id="python-only-distribution">
<h5>Python Only Distribution<a class="headerlink" href="#python-only-distribution" title="Permalink to this heading">¶</a></h5>
<p>There are multiple features of the library which require C++ components to be enabled. This includes both the TorchScript frontend which accepts TorchScript modules for compilation
and the Torch-TensorRT runtime, the default executor for modules compiled with Torch-TensorRT, be it with the TorchScript or Dynamo frontend.</p>
<p>In the case you may want a build which does not require C++ you can disable these features and avoid building these components. As a result, the only available runtime will be the Python based on
which has implications for features like serialization.</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span><span class="nv">PYTHON_ONLY</span><span class="o">=</span><span class="m">1</span><span class="w"> </span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>--pre<span class="w"> </span>.<span class="w"> </span>--extra-index-url<span class="w"> </span>https://download.pytorch.org/whl/nightly/cu130
</pre></div>
</div>
</section>
<section id="no-torchscript-frontend">
<h5>No TorchScript Frontend<a class="headerlink" href="#no-torchscript-frontend" title="Permalink to this heading">¶</a></h5>
<p>The TorchScript frontend is a legacy feature of Torch-TensorRT which is now in maintenance as TorchDynamo has become the preferred compiler technology for this project. It contains quite a bit
of C++ code that is no longer necessary for most users. Therefore you can exclude this component from your build to speed up build times. The C++ based runtime will still be available to use.</p>
<div class="highlight-sh notranslate"><div class="highlight"><pre><span></span><span class="nv">NO_TORCHSCRIPT</span><span class="o">=</span><span class="m">1</span><span class="w"> </span>python<span class="w"> </span>-m<span class="w"> </span>pip<span class="w"> </span>install<span class="w"> </span>--pre<span class="w"> </span>.<span class="w"> </span>--extra-index-url<span class="w"> </span>https://download.pytorch.org/whl/nightly/cu130
</pre></div>
</div>
</section>
</section>
<section id="building-the-c-library-standalone-torchscript-only">
<h4>Building the C++ Library Standalone (TorchScript Only)<a class="headerlink" href="#building-the-c-library-standalone-torchscript-only" title="Permalink to this heading">¶</a></h4>
<section id="release-build">
<h5>Release Build<a class="headerlink" href="#release-build" title="Permalink to this heading">¶</a></h5>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>bazel<span class="w"> </span>build<span class="w"> </span>//:libtorchtrt<span class="w"> </span>-c<span class="w"> </span>opt
</pre></div>
</div>
<p>A tarball with the include files and library can then be found in <code class="docutils literal notranslate"><span class="pre">bazel-bin</span></code></p>
</section>
<section id="debug-build">
<span id="build-from-archive-debug"></span><h5>Debug Build<a class="headerlink" href="#debug-build" title="Permalink to this heading">¶</a></h5>
<p>To build with debug symbols use the following command</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>bazel<span class="w"> </span>build<span class="w"> </span>//:libtorchtrt<span class="w"> </span>-c<span class="w"> </span>dbg
</pre></div>
</div>
<p>A tarball with the include files and library can then be found in <code class="docutils literal notranslate"><span class="pre">bazel-bin</span></code></p>
</section>
</section>
<section id="choosing-the-right-abi">
<span id="abis"></span><h4>Choosing the Right ABI<a class="headerlink" href="#choosing-the-right-abi" title="Permalink to this heading">¶</a></h4>
<p>For the old versions, there were two ABI options to compile Torch-TensorRT which were incompatible with each other,
pre-cxx11-abi and cxx11-abi. The complexity came from the different distributions of PyTorch. Fortunately, PyTorch
has switched to cxx11-abi for all distributions. Below is a table with general pairings of PyTorch distribution
sources and the recommended commands:</p>
<table class="docutils align-default">
<colgroup>
<col style="width: 33%" />
<col style="width: 31%" />
<col style="width: 36%" />
</colgroup>
<thead>
<tr class="row-odd"><th class="head"><p>PyTorch Source</p></th>
<th class="head"><p>Recommended Python Compilation Command</p></th>
<th class="head"><p>Recommended C++ Compilation Command</p></th>
</tr>
</thead>
<tbody>
<tr class="row-even"><td><p>PyTorch whl file from PyTorch.org</p></td>
<td><p>python -m pip install .</p></td>
<td><p>bazel build //:libtorchtrt -c opt</p></td>
</tr>
<tr class="row-odd"><td><p>libtorch-cxx11-abi-shared-with-deps-<a href="#id3"><span class="problematic" id="id4">*</span></a>.zip from PyTorch.org</p></td>
<td><p>python setup.py bdist_wheel</p></td>
<td><p>bazel build //:libtorchtrt -c opt</p></td>
</tr>
<tr class="row-even"><td><p>PyTorch preinstalled in an NGC container</p></td>
<td><p>python setup.py bdist_wheel</p></td>
<td><p>bazel build //:libtorchtrt -c opt</p></td>
</tr>
<tr class="row-odd"><td><p>PyTorch from the NVIDIA Forums for Jetson</p></td>
<td><p>python setup.py bdist_wheel</p></td>
<td><p>bazel build //:libtorchtrt -c opt</p></td>
</tr>
<tr class="row-even"><td><p>PyTorch built from Source</p></td>
<td><p>python setup.py bdist_wheel</p></td>
<td><p>bazel build //:libtorchtrt -c opt</p></td>
</tr>
</tbody>
</table>
<blockquote>
<div><p>NOTE: For all of the above cases you must correctly declare the source of PyTorch you intend to use in your WORKSPACE file for both Python and C++ builds. See below for more information</p>
</div></blockquote>
</section>
</section>
<section id="building-on-windows">
<h3>Building on Windows<a class="headerlink" href="#building-on-windows" title="Permalink to this heading">¶</a></h3>
<ul class="simple">
<li><p>Microsoft VS 2022 Tools</p></li>
<li><p>Bazelisk</p></li>
<li><p>CUDA</p></li>
</ul>
<section id="build-steps">
<h4>Build steps<a class="headerlink" href="#build-steps" title="Permalink to this heading">¶</a></h4>
<ul class="simple">
<li><p>Open the app “x64 Native Tools Command Prompt for VS 2022” - note that Admin privileges may be necessary</p></li>
<li><p>Ensure Bazelisk (Bazel launcher) is installed on your machine and available from the command line. Package installers such as Chocolatey can be used to install Bazelisk</p></li>
<li><p>Install latest version of Torch (i.e. with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--pre</span> <span class="pre">torch</span> <span class="pre">--index-url</span> <span class="pre">https://download.pytorch.org/whl/nightly/cu130</span></code>)</p></li>
<li><p>Clone the Torch-TensorRT repository and navigate to its root directory</p></li>
<li><p>Run <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">ninja</span> <span class="pre">wheel</span> <span class="pre">setuptools</span></code></p></li>
<li><p>Run <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--pre</span> <span class="pre">-r</span> <span class="pre">py/requirements.txt</span></code></p></li>
<li><p>Run <code class="docutils literal notranslate"><span class="pre">set</span> <span class="pre">DISTUTILS_USE_SDK=1</span></code></p></li>
<li><p>Run <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">setup.py</span> <span class="pre">bdist_wheel</span></code></p></li>
<li><p>Run <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">dist/*.whl</span></code></p></li>
</ul>
</section>
<section id="advanced-setup-and-troubleshooting">
<h4>Advanced setup and Troubleshooting<a class="headerlink" href="#advanced-setup-and-troubleshooting" title="Permalink to this heading">¶</a></h4>
<p>In the <code class="docutils literal notranslate"><span class="pre">WORKSPACE</span></code> file, the <code class="docutils literal notranslate"><span class="pre">cuda_win</span></code>, <code class="docutils literal notranslate"><span class="pre">libtorch_win</span></code>, and <code class="docutils literal notranslate"><span class="pre">tensorrt_win</span></code> are Windows-specific modules which can be customized. For instance, if you would like to build with a different version of CUDA, or your CUDA installation is in a non-standard location, update the <cite>path</cite> in the <cite>cuda_win</cite> module.</p>
<p>Similarly, if you would like to use a different version of pytorch or tensorrt, customize the <cite>urls</cite> in the <code class="docutils literal notranslate"><span class="pre">libtorch_win</span></code> and <code class="docutils literal notranslate"><span class="pre">tensorrt_win</span></code> modules, respectively.</p>
<p>Local versions of these packages can also be used on Windows. See <code class="docutils literal notranslate"><span class="pre">toolchains\\ci_workspaces\\WORKSPACE.win.release.tmpl</span></code> for an example of using a local version of TensorRT on Windows.</p>
</section>
</section>
<section id="alternative-build-systems">
<h3>Alternative Build Systems<a class="headerlink" href="#alternative-build-systems" title="Permalink to this heading">¶</a></h3>
<section id="building-with-cmake-torchscript-only">
<h4>Building with CMake (TorchScript Only)<a class="headerlink" href="#building-with-cmake-torchscript-only" title="Permalink to this heading">¶</a></h4>
<p>It is possible to build the API libraries (in cpp/) and the torchtrtc executable using CMake instead of Bazel.
Currently, the python API and the tests cannot be built with CMake.
Begin by installing CMake.</p>
<blockquote>
<div><ul class="simple">
<li><p>Latest releases of CMake and instructions on how to install are available for different platforms
[on their website](<a class="reference external" href="https://cmake.org/download/">https://cmake.org/download/</a>).</p></li>
</ul>
</div></blockquote>
<p>A few useful CMake options include:</p>
<blockquote>
<div><ul class="simple">
<li><p>CMake finders for TensorRT are provided in <cite>cmake/Modules</cite>. In order for CMake to use them, pass
<cite>-DCMAKE_MODULE_PATH=cmake/Modules</cite> when configuring the project with CMake.</p></li>
<li><p>Libtorch provides its own CMake finder. In case CMake doesn’t find it, pass the path to your install of
libtorch with <cite>-DTorch_DIR=&lt;path to libtorch&gt;/share/cmake/Torch</cite></p></li>
<li><p>If TensorRT is not found with the provided cmake finder, specify <cite>-DTensorRT_ROOT=&lt;path to TensorRT&gt;</cite></p></li>
<li><p>Finally, configure and build the project in a build directory of your choice with the following command
from the root of Torch-TensorRT project:</p></li>
</ul>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>cmake<span class="w"> </span>-S.<span class="w"> </span>-B&lt;build<span class="w"> </span>directory&gt;<span class="w"> </span><span class="se">\</span>
<span class="w">    </span><span class="o">[</span>-DCMAKE_MODULE_PATH<span class="o">=</span>cmake/Module<span class="o">]</span><span class="w"> </span><span class="se">\</span>
<span class="w">    </span><span class="o">[</span>-DTorch_DIR<span class="o">=</span>&lt;path<span class="w"> </span>to<span class="w"> </span>libtorch&gt;/share/cmake/Torch<span class="o">]</span><span class="w"> </span><span class="se">\</span>
<span class="w">    </span><span class="o">[</span>-DTensorRT_ROOT<span class="o">=</span>&lt;path<span class="w"> </span>to<span class="w"> </span>TensorRT&gt;<span class="o">]</span><span class="w"> </span><span class="se">\</span>
<span class="w">    </span><span class="o">[</span>-DCMAKE_BUILD_TYPE<span class="o">=</span>Debug<span class="p">|</span>Release<span class="o">]</span>
cmake<span class="w"> </span>--build<span class="w"> </span>&lt;build<span class="w"> </span>directory&gt;
</pre></div>
</div>
</div></blockquote>
</section>
<section id="building-natively-on-aarch64-jetson">
<h4>Building Natively on aarch64 (Jetson)<a class="headerlink" href="#building-natively-on-aarch64-jetson" title="Permalink to this heading">¶</a></h4>
<section id="prerequisites">
<h5>Prerequisites<a class="headerlink" href="#prerequisites" title="Permalink to this heading">¶</a></h5>
<p>Install or compile a build of PyTorch/LibTorch for aarch64</p>
<p>NVIDIA hosts builds the latest release branch for Jetson here:</p>
<blockquote>
<div><p><a class="reference external" href="https://forums.developer.nvidia.com/t/pytorch-for-jetson-version-1-10-now-available/72048">https://forums.developer.nvidia.com/t/pytorch-for-jetson-version-1-10-now-available/72048</a></p>
</div></blockquote>
</section>
<section id="environment-setup">
<h5>Environment Setup<a class="headerlink" href="#environment-setup" title="Permalink to this heading">¶</a></h5>
<p>To build natively on aarch64-linux-gnu platform, configure the <code class="docutils literal notranslate"><span class="pre">WORKSPACE</span></code> with local available dependencies.</p>
<ol class="arabic">
<li><p>Replace <code class="docutils literal notranslate"><span class="pre">WORKSPACE</span></code> with the corresponding WORKSPACE file in <code class="docutils literal notranslate"><span class="pre">//toolchains/jp_workspaces</span></code></p></li>
<li><p>Configure the correct paths to directory roots containing local dependencies in the <code class="docutils literal notranslate"><span class="pre">new_local_repository</span></code> rules:</p>
<blockquote>
<div><p>NOTE: If you installed PyTorch using a pip package, the correct path is the path to the root of the python torch package.
In the case that you installed with <code class="docutils literal notranslate"><span class="pre">sudo</span> <span class="pre">pip</span> <span class="pre">install</span></code> this will be <code class="docutils literal notranslate"><span class="pre">/usr/local/lib/python3.8/dist-packages/torch</span></code>.
In the case you installed with <code class="docutils literal notranslate"><span class="pre">pip</span> <span class="pre">install</span> <span class="pre">--user</span></code> this will be <code class="docutils literal notranslate"><span class="pre">$HOME/.local/lib/python3.8/site-packages/torch</span></code>.</p>
</div></blockquote>
</li>
</ol>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>new_local_repository<span class="o">(</span>
<span class="w">    </span><span class="nv">name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;libtorch&quot;</span>,
<span class="w">    </span><span class="nv">path</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;/usr/local/lib/python3.8/dist-packages/torch&quot;</span>,
<span class="w">    </span><span class="nv">build_file</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s2">&quot;third_party/libtorch/BUILD&quot;</span>
<span class="o">)</span>
</pre></div>
</div>
</section>
<section id="compile-c-library-and-compiler-cli">
<h5>Compile C++ Library and Compiler CLI<a class="headerlink" href="#compile-c-library-and-compiler-cli" title="Permalink to this heading">¶</a></h5>
<blockquote>
<div><p>NOTE: Due to shifting dependency locations between Jetpack 4.5 and 4.6 there is a now a flag to inform bazel of the Jetpack version</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>--platforms<span class="w"> </span>//toolchains:jetpack_x.x
</pre></div>
</div>
</div></blockquote>
<p>Compile Torch-TensorRT library using bazel command:</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>bazel<span class="w"> </span>build<span class="w"> </span>//:libtorchtrt<span class="w"> </span>--platforms<span class="w"> </span>//toolchains:jetpack_5.0
</pre></div>
</div>
</section>
<section id="compile-python-api">
<h5>Compile Python API<a class="headerlink" href="#compile-python-api" title="Permalink to this heading">¶</a></h5>
<blockquote>
<div><p>NOTE: Due to shifting dependencies locations between Jetpack 4.5 and newer Jetpack versions there is now a flag for <code class="docutils literal notranslate"><span class="pre">setup.py</span></code> which sets the jetpack version (default: 5.0)</p>
</div></blockquote>
<p>Compile the Python API using the following command from the <code class="docutils literal notranslate"><span class="pre">//py</span></code> directory:</p>
<div class="highlight-shell notranslate"><div class="highlight"><pre><span></span>python3<span class="w"> </span>setup.py<span class="w"> </span>install
</pre></div>
</div>
<p>If you are building for Jetpack 4.5 add the <code class="docutils literal notranslate"><span class="pre">--jetpack-version</span> <span class="pre">5.0</span></code> flag</p>
</section>
</section>
</section>
</section>
</section>


             </article>
             
            </div>
            <footer>
  
    <div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
      
        <a href="jetpack.html" class="btn btn-neutral float-right" title="Torch-TensorRT in JetPack" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
      
      
        <a href="../index.html" class="btn btn-neutral" title="Torch-TensorRT" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
      
    </div>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2024, NVIDIA Corporation.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              <ul>
<li><a class="reference internal" href="#">Installation</a><ul>
<li><a class="reference internal" href="#precompiled-binaries">Precompiled Binaries</a><ul>
<li><a class="reference internal" href="#dependencies">Dependencies</a></li>
<li><a class="reference internal" href="#installing-torch-tensorrt">Installing Torch-TensorRT</a></li>
<li><a class="reference internal" href="#installing-torch-tensorrt-for-a-specific-cuda-version">Installing Torch-TensorRT for a specific CUDA version</a></li>
<li><a class="reference internal" href="#installing-nightly-builds">Installing Nightly Builds</a></li>
<li><a class="reference internal" href="#c-precompiled-binaries-torchscript-only">C++ Precompiled Binaries (TorchScript Only)</a></li>
</ul>
</li>
<li><a class="reference internal" href="#compiling-from-source">Compiling From Source</a><ul>
<li><a class="reference internal" href="#building-on-linux">Building on Linux</a><ul>
<li><a class="reference internal" href="#installing-deps">Dependencies</a></li>
<li><a class="reference internal" href="#building-the-package">Building the Package</a></li>
<li><a class="reference internal" href="#additional-build-options">Additional Build Options</a><ul>
<li><a class="reference internal" href="#python-only-distribution">Python Only Distribution</a></li>
<li><a class="reference internal" href="#no-torchscript-frontend">No TorchScript Frontend</a></li>
</ul>
</li>
<li><a class="reference internal" href="#building-the-c-library-standalone-torchscript-only">Building the C++ Library Standalone (TorchScript Only)</a><ul>
<li><a class="reference internal" href="#release-build">Release Build</a></li>
<li><a class="reference internal" href="#debug-build">Debug Build</a></li>
</ul>
</li>
<li><a class="reference internal" href="#choosing-the-right-abi">Choosing the Right ABI</a></li>
</ul>
</li>
<li><a class="reference internal" href="#building-on-windows">Building on Windows</a><ul>
<li><a class="reference internal" href="#build-steps">Build steps</a></li>
<li><a class="reference internal" href="#advanced-setup-and-troubleshooting">Advanced setup and Troubleshooting</a></li>
</ul>
</li>
<li><a class="reference internal" href="#alternative-build-systems">Alternative Build Systems</a><ul>
<li><a class="reference internal" href="#building-with-cmake-torchscript-only">Building with CMake (TorchScript Only)</a></li>
<li><a class="reference internal" href="#building-natively-on-aarch64-jetson">Building Natively on aarch64 (Jetson)</a><ul>
<li><a class="reference internal" href="#prerequisites">Prerequisites</a></li>
<li><a class="reference internal" href="#environment-setup">Environment Setup</a></li>
<li><a class="reference internal" href="#compile-c-library-and-compiler-cli">Compile C++ Library and Compiler CLI</a></li>
<li><a class="reference internal" href="#compile-python-api">Compile Python API</a></li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>
</li>
</ul>

            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
         <script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
         <script src="../_static/jquery.js"></script>
         <script src="../_static/underscore.js"></script>
         <script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
         <script src="../_static/doctools.js"></script>
         <script src="../_static/collapsible-lists/js/CollapsibleLists.compressed.js"></script>
         <script src="../_static/collapsible-lists/js/apply-collapsible-lists.js"></script>
         <script crossorigin="anonymous" integrity="sha256-Ae2Vz/4ePdIu6ZyI/5ZGsYnb+m0JlOmKPjt6XZ9JJkA=" src="https://cdnjs.cloudflare.com/ajax/libs/require.js/2.3.4/require.min.js"></script>
     

  

  <script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script> 

  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">Stay up to date</li>
            <li><a href="https://www.facebook.com/pytorch" target="_blank">Facebook</a></li>
            <li><a href="https://twitter.com/pytorch" target="_blank">Twitter</a></li>
            <li><a href="https://www.youtube.com/pytorch" target="_blank">YouTube</a></li>
            <li><a href="https://www.linkedin.com/company/pytorch" target="_blank">LinkedIn</a></li>
          </ul>  
          </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title">PyTorch Podcasts</li>
            <li><a href="https://open.spotify.com/show/6UzHKeiy368jKfQMKKvJY5" target="_blank">Spotify</a></li>
            <li><a href="https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008" target="_blank">Apple</a></li>
            <li><a href="https://www.google.com/podcasts?feed=aHR0cHM6Ly9mZWVkcy5zaW1wbGVjYXN0LmNvbS9PQjVGa0lsOA%3D%3D" target="_blank">Google</a></li>
            <li><a href="https://music.amazon.com/podcasts/7a4e6f0e-26c2-49e9-a478-41bd244197d0/PyTorch-Developer-Podcast?" target="_blank">Amazon</a></li>
          </ul>
         </div>
        </div>
        
        <div class="privacy-policy">
          <ul>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/terms/" target="_blank">Terms</a></li>
            <li class="privacy-policy-links">|</li>
            <li class="privacy-policy-links"><a href="https://www.linuxfoundation.org/privacy-policy/" target="_blank">Privacy</a></li>
          </ul>
        </div>
        <div class="copyright">
        <p>© Copyright The Linux Foundation. The PyTorch Foundation is a project of The Linux Foundation.
          For web site terms of use, trademark policy and other policies applicable to The PyTorch Foundation please see
          <a href="https://www.linuxfoundation.org/policies/">www.linuxfoundation.org/policies/</a>. The PyTorch Foundation supports the PyTorch open source
          project, which has been established as PyTorch Project a Series of LF Projects, LLC. For policies applicable to the PyTorch Project a Series of LF Projects, LLC,
          please see <a href="https://www.lfprojects.org/policies/">www.lfprojects.org/policies/</a>.</p>
      </div>
     </div>

  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
           <li class="resources-mobile-menu-title">
             <a>Learn</a>
           </li>
           <ul class="resources-mobile-menu-items">
             <li>
               <a href="https://pytorch.org/get-started">Get Started</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials">Tutorials</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials/beginner/basics/intro.html">Learn the Basics</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials/recipes/recipes_index.html">PyTorch Recipes</a>
             </li>
             <li>
               <a href="https://pytorch.org/tutorials/beginner/introyt.html">Introduction to PyTorch - YouTube Series</a>
             </li>
           </ul>
           <li class="resources-mobile-menu-title">
             <a>Ecosystem</a>
           </li>
           <ul class="resources-mobile-menu-items">
             <li>
               <a href="https://pytorch.org/ecosystem">Tools</a>
             </li>
             <li>
               <a href="https://pytorch.org/#community-module">Community</a>
             </li>
             <li>
               <a href="https://discuss.pytorch.org/">Forums</a>
             </li>
             <li>
               <a href="https://pytorch.org/resources">Developer Resources</a>
             </li>
             <li>
               <a href="https://pytorch.org/ecosystem/contributor-awards-2023">Contributor Awards - 2024</a>
             </li>
           </ul>

           <li class="resources-mobile-menu-title">
             <a>Edge</a>
           </li>

           <ul class="resources-mobile-menu-items">
             <li>
               <a href="https://pytorch.org/edge">About PyTorch Edge</a>
             </li>
             
             <li>
               <a href="https://pytorch.org/executorch-overview">ExecuTorch</a>
             </li>
             <li>
               <a href="https://pytorch.org/executorch/stable/index.html">ExecuTorch Documentation</a>
             </li>
           </ul>

           <li class="resources-mobile-menu-title">
             <a>Docs</a>
           </li>

           <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/docs/stable/index.html">PyTorch</a>
            </li>

            <li>
              <a href="https://pytorch.org/pytorch-domains">PyTorch Domains</a>
            </li>
          </ul>

          <li class="resources-mobile-menu-title">
            <a>Blog & News</a>
          </li>
            
           <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/blog/">PyTorch Blog</a>
            </li>
            <li>
              <a href="https://pytorch.org/community-blog">Community Blog</a>
            </li>

            <li>
              <a href="https://pytorch.org/videos">Videos</a>
            </li>

            <li>
              <a href="https://pytorch.org/community-stories">Community Stories</a>
            </li>
            <li>
              <a href="https://pytorch.org/events">Events</a>
            </li>
            <li>
               <a href="https://pytorch.org/newsletter">Newsletter</a>
             </li>
          </ul>
          
          <li class="resources-mobile-menu-title">
            <a>About</a>
          </li>

          <ul class="resources-mobile-menu-items">
            <li>
              <a href="https://pytorch.org/foundation">PyTorch Foundation</a>
            </li>
            <li>
              <a href="https://pytorch.org/governing-board">Governing Board</a>
            </li>
            <li>
               <a href="https://pytorch.org/credits">Cloud Credit Program</a>
            </li>
            <li>
               <a href="https://pytorch.org/tac">Technical Advisory Council</a>
            </li>
            <li>
               <a href="https://pytorch.org/staff">Staff</a>
            </li>
            <li>
               <a href="https://pytorch.org/contact-us">Contact Us</a>
            </li>
          </ul>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>